/* Complet (Autocompletion through Graph Databases)
 *  Copyright (C) 2012  Paul Wagner, Till Speicher
 *
 *  This program is free software; you can redistribute it and/or modify it
 *  under the terms of the GNU General Public License as published by the
 *  Free Software Foundation; either version 3 of the License, or (at your
 *  option) any later version.
 *
 *  This program is distributed in the hope that it will be useful, but
 *  WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
 *  or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
 *  for more details.
 *
 *  You should have received a copy of the GNU General Public License along
 *  with this program; if not, see <http:/* www.gnu.org/licenses/>.
*/

/*
 * Parser for Google N-Gram files
 *
 * @version 1.3
 * @author	Paul Wagner
 *
 */
package complet.app.io.ngramparser;

import complet.helper.ConfigHelper;
import complet.helper.IOHelper;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;

public class NGramParser {

    private static String overhead = "";

    public static void main(final String[] args) {
        ConfigHelper.IN_PATH = "S:\\googlengrams\\5_neu\\googlebooks-ger-all-5gram-20090715-[C].csv";
        ConfigHelper.OUT_PATH = "E:\\paul\\Dokumente\\MyDocuments\\NetBeans Workspace\\Completion\\inputfiles\\ngrams\\5grams_ucsc.txt";
        if (args.length > 0) {
            ConfigHelper.loadConfigFile(args[0]);
        }
        try {
            int counter = 0;
            BufferedReader in = null;
            FileWriter out = null;
            String line = null;
            File file = new File(ConfigHelper.IN_PATH.replace("[C]", new Integer(counter).toString()));
            File out_file = new File(ConfigHelper.OUT_PATH);
            while (file.exists()) {
                IOHelper.pln("Processing NGram: " + file.getName());
                try {
                    in = new BufferedReader(new FileReader(file));
                    out = new FileWriter(out_file, true);
                } catch (Exception ex) {
                    IOHelper.logError("(NGramParser) Failed to open file, " + ex.getMessage());
                }
                try {
                    while ((line = getLine(in)) != null) {
                        if (!line.isEmpty()) {
                            out.write(line);
                            out.write(System.getProperty("line.separator"));
                            out.flush();
                        }
                    }
                } catch (IOException ex) {
                    IOHelper.logError("(NGramParser) Failed to write file, " + ex.getMessage());
                }
                counter++;
                overhead = "";
                file = new File(ConfigHelper.IN_PATH.replace("[C]", new Integer(counter).toString()));
            }
            //out.flush();
            out.close();
        } catch (IOException ex) {
            IOHelper.logError("(NGramParser) Failed to parse file, " + ex.getMessage());
        }
    }

    private static String getLine(BufferedReader in) {
        String line, out = "";
        int count = 0;
        if (overhead == null) {
            return null;
        }
        try {
            out = overhead;
            while ((line = in.readLine()) != null && line.split("\t")[0].equals(overhead)) {
                try {
                    count += Integer.parseInt(line.split("\t")[2]);
                } catch (Exception e) {
                    return ""; // Line seems to be in an invalid format, skip
                }
            }
            if (line != null) {
                overhead = line.split("\t")[0];
            } else {
                overhead = null;
            }
        } catch (IOException ex) {
            IOHelper.logError("(NGramParser) Failed to read file, " + ex.getMessage());
        }

//        out = out.replaceAll("ß", "ss");
//        out = out.replaceAll("ü", "ue");
//        out = out.replaceAll("ä", "ae");
//        out = out.replaceAll("ö", "oe");
        out = out.replaceAll("[^a-zA-ZßäöüÄÖÜ ]", "");
        while (out.contains("  ")) {
            out = out.replaceAll("  ", " ");
        }
        out = out.trim();
        if (out.isEmpty()) {
//        if (out.isEmpty() || out.split(" ").length <= 1) {
            return "";
        }

        return out + "\t" + count;
    }
}
